{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/cql/anaconda3/lib/python3.6/site-packages/h5py/__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n",
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import time\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "from bert4keras.backend import keras, set_gelu\n",
    "from bert4keras.tokenizers import Tokenizer\n",
    "from bert4keras.models import build_transformer_model\n",
    "from bert4keras.optimizers import Adam, extend_with_piecewise_linear_lr\n",
    "from bert4keras.snippets import sequence_padding, DataGenerator\n",
    "from bert4keras.snippets import open\n",
    "from keras.layers import Lambda, Dense"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "os.environ['CUDA_VISIBLE_DEVICES'] = '0'\n",
    "set_gelu('tanh')  # 切换gelu版本"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 加载数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_data(filename):\n",
    "    df = pd.read_csv(filename, header=0, encoding='utf8')\n",
    "    f = df[['char','class']].values\n",
    "    D = []\n",
    "    \n",
    "    for i,l in enumerate(f):\n",
    "        text, label = l\n",
    "        D.append((text, int(label)))\n",
    "    return D"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data = load_data('data/train.csv')\n",
    "valid_data = load_data('data/val.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((164151, 2), (29776, 2))"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.array(train_data).shape, np.array(valid_data).shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 微调Bert_rbtl3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# bert rbtl3\n",
    "num_classes = 26\n",
    "maxlen = 180\n",
    "batch_size = 128\n",
    "model_type = 'bert'\n",
    "best_model = './model/bert/bert_rbtl3_2.weights'\n",
    "\n",
    "config_path = '/data/models/embedding/chinese_rbtl3_L-3_H-1024_A-16/bert_config_rbtl3.json'\n",
    "checkpoint_path = '/data/models/embedding/chinese_rbtl3_L-3_H-1024_A-16/bert_model.ckpt'\n",
    "dict_path = '/data/models/embedding/chinese_rbtl3_L-3_H-1024_A-16/vocab.txt'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/cql/anaconda3/lib/python3.6/site-packages/tensorflow_core/python/ops/resource_variable_ops.py:1630: calling BaseResourceVariable.__init__ (from tensorflow.python.ops.resource_variable_ops) with constraint is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "If using Keras pass *_constraint arguments to layers.\n",
      "Model: \"model_2\"\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "Input-Token (InputLayer)        (None, None)         0                                            \n",
      "__________________________________________________________________________________________________\n",
      "Input-Segment (InputLayer)      (None, None)         0                                            \n",
      "__________________________________________________________________________________________________\n",
      "Embedding-Token (Embedding)     (None, None, 1024)   21635072    Input-Token[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "Embedding-Segment (Embedding)   (None, None, 1024)   2048        Input-Segment[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "Embedding-Token-Segment (Add)   (None, None, 1024)   0           Embedding-Token[0][0]            \n",
      "                                                                 Embedding-Segment[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "Embedding-Position (PositionEmb (None, None, 1024)   524288      Embedding-Token-Segment[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "Embedding-Norm (LayerNormalizat (None, None, 1024)   2048        Embedding-Position[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "Embedding-Dropout (Dropout)     (None, None, 1024)   0           Embedding-Norm[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "Transformer-0-MultiHeadSelfAtte (None, None, 1024)   4198400     Embedding-Dropout[0][0]          \n",
      "                                                                 Embedding-Dropout[0][0]          \n",
      "                                                                 Embedding-Dropout[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "Transformer-0-MultiHeadSelfAtte (None, None, 1024)   0           Transformer-0-MultiHeadSelfAttent\n",
      "__________________________________________________________________________________________________\n",
      "Transformer-0-MultiHeadSelfAtte (None, None, 1024)   0           Embedding-Dropout[0][0]          \n",
      "                                                                 Transformer-0-MultiHeadSelfAttent\n",
      "__________________________________________________________________________________________________\n",
      "Transformer-0-MultiHeadSelfAtte (None, None, 1024)   2048        Transformer-0-MultiHeadSelfAttent\n",
      "__________________________________________________________________________________________________\n",
      "Transformer-0-FeedForward (Feed (None, None, 1024)   8393728     Transformer-0-MultiHeadSelfAttent\n",
      "__________________________________________________________________________________________________\n",
      "Transformer-0-FeedForward-Dropo (None, None, 1024)   0           Transformer-0-FeedForward[0][0]  \n",
      "__________________________________________________________________________________________________\n",
      "Transformer-0-FeedForward-Add ( (None, None, 1024)   0           Transformer-0-MultiHeadSelfAttent\n",
      "                                                                 Transformer-0-FeedForward-Dropout\n",
      "__________________________________________________________________________________________________\n",
      "Transformer-0-FeedForward-Norm  (None, None, 1024)   2048        Transformer-0-FeedForward-Add[0][\n",
      "__________________________________________________________________________________________________\n",
      "Transformer-1-MultiHeadSelfAtte (None, None, 1024)   4198400     Transformer-0-FeedForward-Norm[0]\n",
      "                                                                 Transformer-0-FeedForward-Norm[0]\n",
      "                                                                 Transformer-0-FeedForward-Norm[0]\n",
      "__________________________________________________________________________________________________\n",
      "Transformer-1-MultiHeadSelfAtte (None, None, 1024)   0           Transformer-1-MultiHeadSelfAttent\n",
      "__________________________________________________________________________________________________\n",
      "Transformer-1-MultiHeadSelfAtte (None, None, 1024)   0           Transformer-0-FeedForward-Norm[0]\n",
      "                                                                 Transformer-1-MultiHeadSelfAttent\n",
      "__________________________________________________________________________________________________\n",
      "Transformer-1-MultiHeadSelfAtte (None, None, 1024)   2048        Transformer-1-MultiHeadSelfAttent\n",
      "__________________________________________________________________________________________________\n",
      "Transformer-1-FeedForward (Feed (None, None, 1024)   8393728     Transformer-1-MultiHeadSelfAttent\n",
      "__________________________________________________________________________________________________\n",
      "Transformer-1-FeedForward-Dropo (None, None, 1024)   0           Transformer-1-FeedForward[0][0]  \n",
      "__________________________________________________________________________________________________\n",
      "Transformer-1-FeedForward-Add ( (None, None, 1024)   0           Transformer-1-MultiHeadSelfAttent\n",
      "                                                                 Transformer-1-FeedForward-Dropout\n",
      "__________________________________________________________________________________________________\n",
      "Transformer-1-FeedForward-Norm  (None, None, 1024)   2048        Transformer-1-FeedForward-Add[0][\n",
      "__________________________________________________________________________________________________\n",
      "Transformer-2-MultiHeadSelfAtte (None, None, 1024)   4198400     Transformer-1-FeedForward-Norm[0]\n",
      "                                                                 Transformer-1-FeedForward-Norm[0]\n",
      "                                                                 Transformer-1-FeedForward-Norm[0]\n",
      "__________________________________________________________________________________________________\n",
      "Transformer-2-MultiHeadSelfAtte (None, None, 1024)   0           Transformer-2-MultiHeadSelfAttent\n",
      "__________________________________________________________________________________________________\n",
      "Transformer-2-MultiHeadSelfAtte (None, None, 1024)   0           Transformer-1-FeedForward-Norm[0]\n",
      "                                                                 Transformer-2-MultiHeadSelfAttent\n",
      "__________________________________________________________________________________________________\n",
      "Transformer-2-MultiHeadSelfAtte (None, None, 1024)   2048        Transformer-2-MultiHeadSelfAttent\n",
      "__________________________________________________________________________________________________\n",
      "Transformer-2-FeedForward (Feed (None, None, 1024)   8393728     Transformer-2-MultiHeadSelfAttent\n",
      "__________________________________________________________________________________________________\n",
      "Transformer-2-FeedForward-Dropo (None, None, 1024)   0           Transformer-2-FeedForward[0][0]  \n",
      "__________________________________________________________________________________________________\n",
      "Transformer-2-FeedForward-Add ( (None, None, 1024)   0           Transformer-2-MultiHeadSelfAttent\n",
      "                                                                 Transformer-2-FeedForward-Dropout\n",
      "__________________________________________________________________________________________________\n",
      "Transformer-2-FeedForward-Norm  (None, None, 1024)   2048        Transformer-2-FeedForward-Add[0][\n",
      "__________________________________________________________________________________________________\n",
      "CLS-token (Lambda)              (None, 1024)         0           Transformer-2-FeedForward-Norm[0]\n",
      "__________________________________________________________________________________________________\n",
      "dense_19 (Dense)                (None, 26)           26650       CLS-token[0][0]                  \n",
      "==================================================================================================\n",
      "Total params: 59,978,778\n",
      "Trainable params: 59,978,778\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# 建立分词器\n",
    "tokenizer = Tokenizer(dict_path, do_lower_case=True)\n",
    "\n",
    "class data_generator(DataGenerator):\n",
    "    \"\"\"数据生成器\n",
    "    \"\"\"\n",
    "    def __iter__(self, random=False):\n",
    "        batch_token_ids, batch_segment_ids, batch_labels = [], [], []\n",
    "        for is_end, (text, label) in self.sample(random):\n",
    "            token_ids, segment_ids = tokenizer.encode(text, maxlen=maxlen)\n",
    "            batch_token_ids.append(token_ids)\n",
    "            batch_segment_ids.append(segment_ids)\n",
    "            batch_labels.append([label])\n",
    "            if len(batch_token_ids) == self.batch_size or is_end:\n",
    "                batch_token_ids = sequence_padding(batch_token_ids)\n",
    "                batch_segment_ids = sequence_padding(batch_segment_ids)\n",
    "                batch_labels = sequence_padding(batch_labels)\n",
    "                yield [batch_token_ids, batch_segment_ids], batch_labels\n",
    "                batch_token_ids, batch_segment_ids, batch_labels = [], [], []\n",
    "\n",
    "# 加载预训练模型\n",
    "bert = build_transformer_model(\n",
    "    config_path=config_path,\n",
    "    checkpoint_path=checkpoint_path,\n",
    "    model=model_type,\n",
    "    return_keras_model=False,\n",
    ")\n",
    "\n",
    "# 搭建微调网络\n",
    "output = Lambda(lambda x: x[:, 0], name='CLS-token')(bert.model.output)\n",
    "output = Dense(\n",
    "    units=num_classes,\n",
    "    activation='softmax',\n",
    "    kernel_initializer=bert.initializer\n",
    ")(output)\n",
    "\n",
    "model = keras.models.Model(bert.model.input, output)\n",
    "model.summary()\n",
    "\n",
    "# 派生为带分段线性学习率的优化器。\n",
    "# 其中name参数可选，但最好填入，以区分不同的派生优化器。\n",
    "AdamLR = extend_with_piecewise_linear_lr(Adam, name='AdamLR')\n",
    "\n",
    "model.compile(\n",
    "    loss='sparse_categorical_crossentropy',\n",
    "    # optimizer=Adam(1e-5),  # 用足够小的学习率\n",
    "    optimizer=AdamLR(learning_rate=1e-4, lr_schedule={\n",
    "        1000: 1,\n",
    "        2000: 0.1\n",
    "    }),\n",
    "    metrics=['accuracy'],\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/cql/anaconda3/lib/python3.6/site-packages/tensorflow_core/python/ops/math_grad.py:1424: where (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.where in 2.0, which has the same broadcast rule as np.where\n",
      "WARNING:tensorflow:From /home/cql/anaconda3/lib/python3.6/site-packages/keras/backend/tensorflow_backend.py:422: The name tf.global_variables is deprecated. Please use tf.compat.v1.global_variables instead.\n",
      "\n",
      "Epoch 1/6\n",
      "1283/1283 [==============================] - 763s 594ms/step - loss: 1.1129 - accuracy: 0.6687\n",
      "val_acc: 0.75739, best_val_acc: 0.75739\n",
      "\n",
      "Epoch 2/6\n",
      "1283/1283 [==============================] - 758s 591ms/step - loss: 0.7129 - accuracy: 0.7792\n",
      "val_acc: 0.77848, best_val_acc: 0.77848\n",
      "\n",
      "Epoch 3/6\n",
      "1283/1283 [==============================] - 757s 590ms/step - loss: 0.6089 - accuracy: 0.8111\n",
      "val_acc: 0.77606, best_val_acc: 0.77848\n",
      "\n",
      "Epoch 4/6\n",
      "1283/1283 [==============================] - 756s 590ms/step - loss: 0.5635 - accuracy: 0.8252\n",
      "val_acc: 0.77485, best_val_acc: 0.77848\n",
      "\n",
      "Epoch 5/6\n",
      "1283/1283 [==============================] - 761s 593ms/step - loss: 0.5196 - accuracy: 0.8395\n",
      "val_acc: 0.77515, best_val_acc: 0.77848\n",
      "\n",
      "Epoch 6/6\n",
      "1283/1283 [==============================] - 754s 588ms/step - loss: 0.4743 - accuracy: 0.8539\n",
      "val_acc: 0.77250, best_val_acc: 0.77848\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# 转换数据集\n",
    "train_generator = data_generator(train_data, batch_size)\n",
    "valid_generator = data_generator(valid_data, batch_size)\n",
    "\n",
    "# 定义评估方法\n",
    "def evaluate(data):\n",
    "    total, right = 0., 0.\n",
    "    for x_true, y_true in data:\n",
    "        y_pred = model.predict(x_true).argmax(axis=1)\n",
    "        y_true = y_true[:, 0]\n",
    "        total += len(y_true)\n",
    "        right += (y_true == y_pred).sum()\n",
    "    return right / total\n",
    "\n",
    "class Evaluator(keras.callbacks.Callback):\n",
    "    def __init__(self):\n",
    "        self.best_val_acc = 0.\n",
    "\n",
    "    def on_epoch_end(self, epoch, logs=None):\n",
    "        val_acc = evaluate(valid_generator)\n",
    "        if val_acc > self.best_val_acc:\n",
    "            self.best_val_acc = val_acc\n",
    "            model.save_weights(best_model)\n",
    "        print(\n",
    "            u'val_acc: %.5f, best_val_acc: %.5f\\n' %\n",
    "            (val_acc, self.best_val_acc)\n",
    "        )\n",
    "\n",
    "evaluator = Evaluator()\n",
    "history = model.fit_generator(\n",
    "    train_generator.forfit(),\n",
    "    steps_per_epoch=len(train_generator),\n",
    "    epochs=6,\n",
    "    callbacks=[evaluator]\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "final val acc: 0.778479, cost 57 s\n",
      "\n"
     ]
    }
   ],
   "source": [
    "model.load_weights(best_model)\n",
    "bts = time.time()\n",
    "val_acc = evaluate(valid_generator)\n",
    "print(u'final val acc: %05f, cost %d s\\n' % (val_acc, time.time() - bts))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 测试测试集"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_data = load_data('data/test.csv')\n",
    "test_generator = data_generator(test_data, batch_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "final val acc: 0.765333, cost 97 s\n",
      "\n"
     ]
    }
   ],
   "source": [
    "bts = time.time()\n",
    "test_acc = evaluate(test_generator)\n",
    "print(u'final val acc: %05f, cost %d s\\n' % (test_acc, time.time() - bts))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
